package job.relevance.rawpage;

import java.io.IOException;

import mapred.Job;
import mapred.config.CombineConfig;
import mapred.config.Compress;
import mapred.config.InputConfig;
import mapred.config.JobConfig;
import mapred.config.MapConfig;
import mapred.config.OutputConfig;
import mapred.config.ReduceConfig;
import mapred.example.RemoveDuplicatedReducer;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TextOutputFormat;

public class RLV_RawPage extends Job {

    public RLV_RawPage(String... settings) throws IOException {
        super("RLV Raw Page", settings);
    }

    @Override
    protected CombineConfig getCombineConfig() {
        return CombineConfig.newInstance(RemoveDuplicatedReducer.class);
    }

    @Override
    protected InputConfig getInputConfig(FileSystem fs) throws IOException {
        return InputConfig.newInstance(KeyValueTextInputFormat.class,
                "/user/guojw/SourcePage_NonDuplicated/CJK");
    }

    @Override
    protected MapConfig getMapConfig() {
        return MapConfig.newInstance(RLV_RawPage_Mapper.class, Text.class,
                Text.class);
    }

    @Override
    protected OutputConfig getOutputConfig(FileSystem fs) throws IOException {
        return OutputConfig.newInstance(TextOutputFormat.class, Text.class,
                Text.class, "/user/renp/ChaoGe_RawPage", Compress.NULL);
    }

    @Override
    protected ReduceConfig getReduceConfig() {
        return ReduceConfig.newInstance(RemoveDuplicatedReducer.class, 100);
    }

    @Override
    protected void setUp(JobConfig conf, FileSystem fs) throws IOException {
        conf.set("list", this.getUserSetting("-list"));
    }

    @Override
    protected void tearDown(JobConfig conf, FileSystem fs, RunningJob job)
            throws IOException {
    }

    public static void main(String[] args) throws Exception {
        // Job.buildXML(RLV_RawPage.class);

        new RLV_RawPage(args).submit();
    }

}
